# -*- coding: utf-8 -*-
#!/usr/bin/env python

from fortranformat import FortranRecordReader
import numpy as np
import uuid
from datetime import datetime, timedelta
import time
from metlib.shell.fileutil import *
from littler.models import *
from django.conf import settings
from .prepare_obs_data import *
from metlib.shell.fileutil import BASENAME, DIRNAME, CP, RM
from django.db import connections, transaction

class LittleRReport(object):
    def __init__(self, header, data, end):
        if not header or not data or not end:
            raise ValueError(
                'Input data error! Please check header, data and end!')
        self.point_id = ""
        self.latitude = header[0]  # station latitude (north positive)
        self.longitude = header[1]  # station longitude (east positive)
        self.station_id = header[2].strip()  # ID of station
        self.station_name = header[3].strip()  # Name of station
        # Description of the measurement device
        self.platform = header[4].strip()
        self.source = header[5].strip()  # GTS, NCAR/ADP, BOGUS, etc.
        self.elevation = header[6]  # station elevation (m)
        # Number of valid fields in the report
        self.num_valid_fields = header[7]
        # Number of errors encountered during the decoding of this observation
        self.num_error = header[8]
        # Number of warnings encountered during decoding of this observation
        self.num_warning = header[9]
        self.seq_num = header[10]  # Sequence number of this observation
        # Number of duplicates found for this observation
        self.num_dups = header[11]
        self.is_sound = header[12]  # T/F Multiple levels or a single level
        self.bogus = header[13]  # T/F bogus report or normal one
        # T/F Duplicate and discarded (or merged) report.
        self.discard = header[14]
        self.sut = header[15]  # Seconds since 0000 UTC 1 January 1970
        self.julian = header[16]  # Day of the year
        self.date_char = header[17].strip()  # YYYYMMDDHHmmss
        self.slp = header[18]  # Sea-level pressure (Pa) and a QC flag
        self.slp_qc = header[19]
        # Reference pressure level (for thick-ness) (Pa) and a QC flag
        self.ref_pres = header[20]
        self.ref_pres_qc = header[21]
        self.ground_t = header[22]  # Ground Temperature (T) and QC flag
        self.ground_t_qc = header[23]
        self.sst = header[24]  # Sea-Surface Temperature (K) and QC
        self.sst_qc = header[25]
        self.psfc = header[26]  # Surface pressure (Pa) and QC
        self.psfc_qc = header[27]
        self.precip = header[28]  # Precipitation Accumulation and QC
        self.precip_qc = header[29]
        self.t_max = header[30]  # Daily maximum T (K) and QC
        self.t_max_qc = header[31]
        self.t_min = header[32]  # Daily minimum T (K) and QC
        self.t_min_qc = header[33]
        self.t_min_night = header[34]  # Overnight minimum T (K) and QC
        self.t_min_night_qc = header[35]
        self.p_tend03 = header[36]  # 3-hour pressure change (Pa) and QC
        self.p_tend03_qc = header[37]
        self.p_tend24 = header[38]  # Total cloud cover (oktas) and QC
        self.p_tend24_qc = header[39]
        self.cloud_cvr = header[40]  # Total cloud cover (oktas) and QC
        self.cloud_cvr_qc = header[41]
        self.ceiling = header[42]  # Height (m) of cloud base and QC
        self.ceiling_qc = header[43]
        # process data part
        data_length = len(data) - 1
        if data_length < 1:  # only end report
            self.sound_layers = 0
            self.sound_pressure = None
            self.sound_pressure_qc = None
            self.sound_height = None
            self.sound_height_qc = None
            self.sound_temperature = None
            self.sound_temperature_qc = None
            self.sound_dew_point = None
            self.sound_dew_point_qc = None
            self.sound_speed = None
            self.sound_speed_qc = None
            self.sound_direction = None
            self.sound_direction_qc = None
            self.sound_u = None
            self.sound_u_qc = None
            self.sound_v = None
            self.sound_v_qc = None
            self.sound_rh = None
            self.sound_rh_qc = None
            self.sound_thickness = None
            self.sound_thickness_qc = None
        else:
            data = data[0:-1]
            self.sound_layers = data_length
            self.sound_pressure = np.zeros(data_length)
            self.sound_pressure_qc = np.zeros(data_length)
            self.sound_height = np.zeros(data_length)
            self.sound_height_qc = np.zeros(data_length)
            self.sound_temperature = np.zeros(data_length)
            self.sound_temperature_qc = np.zeros(data_length)
            self.sound_dew_point = np.zeros(data_length)
            self.sound_dew_point_qc = np.zeros(data_length)
            self.sound_speed = np.zeros(data_length)
            self.sound_speed_qc = np.zeros(data_length)
            self.sound_direction = np.zeros(data_length)
            self.sound_direction_qc = np.zeros(data_length)
            self.sound_u = np.zeros(data_length)
            self.sound_u_qc = np.zeros(data_length)
            self.sound_v = np.zeros(data_length)
            self.sound_v_qc = np.zeros(data_length)
            self.sound_rh = np.zeros(data_length)
            self.sound_rh_qc = np.zeros(data_length)
            self.sound_thickness = np.zeros(data_length)
            self.sound_thickness_qc = np.zeros(data_length)
            for i, rec in enumerate(data):
                self.sound_pressure[i] = rec[0]
                self.sound_pressure_qc[i] = rec[1]
                self.sound_height[i] = rec[2]
                self.sound_height_qc[i] = rec[3]
                self.sound_temperature[i] = rec[4]
                self.sound_temperature_qc[i] = rec[5]
                self.sound_dew_point[i] = rec[6]
                self.sound_dew_point_qc[i] = rec[7]
                self.sound_speed[i] = rec[8]
                self.sound_speed_qc[i] = rec[9]
                self.sound_direction[i] = rec[10]
                self.sound_direction_qc[i] = rec[11]
                self.sound_u[i] = rec[12]
                self.sound_u_qc[i] = rec[13]
                self.sound_v[i] = rec[14]
                self.sound_v_qc[i] = rec[15]
                self.sound_rh[i] = rec[16]
                self.sound_rh_qc[i] = rec[17]
                self.sound_thickness[i] = rec[18]
                self.sound_thickness_qc[i] = rec[19]

    def print_all(self):
        print 'latitude=%s, longitude=%s' % (self.latitude, self.longitude)
        print 'station_id=%s, station_name=%s' % (self.station_id, self.station_name)
        print 'platform=%s, source=%s' % (self.platform, self.source)
        print 'elevation=%s' % self.elevation
        print 'num_valid_fields=%s, num_error=%s' % (self.num_valid_fields, self.num_error)
        print 'num_warning=%s, seq_num=%s' % (self.num_warning, self.seq_num)
        print 'num_dups=%s, is_sound=%s' % (self.num_dups, self.is_sound)
        print 'bogus =%s, discard =%s' % (self.bogus, self.discard)
        print 'sut =%s, julian =%s, date_char=%s' % (self.sut, self.julian, self.date_char)
        print 'slp=%s, qc=%s' % (self.slp, self.slp_qc)
        print 'ref_pres=%s, qc=%s' % (self.ref_pres, self.ref_pres_qc)
        print 'ground_t=%s, qc=%s' % (self.ground_t, self.ground_t_qc)
        print 'psfc =%s, qc=%s' % (self.psfc, self.psfc_qc)
        print 'precip=%s, qc=%s' % (self.precip, self.precip_qc)
        print 't_max=%s, qc=%s' % (self.t_max, self.t_max_qc)
        print 't_min=%s, qc=%s' % (self.t_min, self.t_min_qc)
        print 't_min_night=%s, qc=%s' % (self.t_min_night, self.t_min_night_qc)
        print 'cloud_cvr =%s, qc=%s' % (self.cloud_cvr, self.cloud_cvr_qc)
        if self.sound_layers > 0:
            for i in range(self.sound_layers):
                print 'level %s: pres=%s, height=%s, temp=%s, dew=%s, wspd=%s, dir=%s' % (i, self.sound_pressure[i], self.sound_height[i], self.sound_temperature[i], self.sound_dew_point[i], self.sound_speed[i], self.sound_direction[i])

    def set_little_data(self):
        lps = LittlePoint.objects.filter(latitude=self.latitude, longitude=self.longitude)
        if lps.count() == 0:
            LittlePointDict = {}
            point_id = str(uuid.uuid1())
            self.point_id = point_id
            LittlePointDict["point_id"] = point_id
            LittlePointDict["latitude"] = self.latitude
            LittlePointDict["longitude"] = self.longitude
            LittlePointDict["station_id"] = self.station_id
            LittlePointDict["station_name"] = self.station_name
            LittlePointDict["platform"] = self.platform
            LittlePointDict["data_count"] = 1
            date_char = datetime.strptime(self.date_char, "%Y%m%d%H%M%S")
            LittlePointDict["last_data_time"] = date_char
            LittlePointDict["frist_data_time"] = date_char
            LittlePoint.objects.create(**LittlePointDict)
            # point = LittlePoint.objects.filter(latitude=self.latitude,longitude=self.longitude)
        else:
            lp = lps.first()
            point_id = lp.point_id
            self.point_id = lp.point_id
            # lp.data_count = lp.data_count + 1
            # last_data_time = lp.last_data_time
            # frist_data_time = lp.frist_data_time
            # date_char = datetime.strptime(self.date_char, "%Y%m%d%H%M%S")
            # if frist_data_time == None or frist_data_time > date_char:
            #     lp.frist_data_time = date_char
            # if last_data_time == None or last_data_time < date_char:
            #     lp.last_data_time = date_char
            # # # lp.update(data_count=data_count, last_data_time=last_data_time, frist_data_time=frist_data_time)
            # # lp.save()
            # cursor = connection.cursor()
            # cursor.execute(
            #     "update littler_littlepoint set data_count=%s,frist_data_time=%s,last_data_time=%s   where point_id=%s ; " % (
            #     lp.data_count, lp.frist_data_time.strftime("%Y-%m-%d %H:%M:%S"),
            #     lp.last_data_time.strftime("%Y-%m-%d %H:%M:%S"),lp.point_id))
            # transaction.commit_unless_managed()
        LittleDataDict = {}
        data_id = str(uuid.uuid1())
        LittleDataDict["data_id"] = data_id
        LittleDataDict["little_point_id"] = point_id
        LittleDataDict["source"] = self.source
        LittleDataDict["elevation"] = self.elevation
        LittleDataDict["num_valid_fields"] = self.num_valid_fields
        LittleDataDict["seq_num"] = self.seq_num
        LittleDataDict["num_dups"] = self.num_dups
        LittleDataDict["is_sound"] = self.is_sound
        LittleDataDict["bogus"] = self.bogus
        LittleDataDict["discard"] = self.discard
        LittleDataDict["sut"] = self.sut
        LittleDataDict["julian"] = self.julian
        LittleDataDict["date_char"] = datetime.strptime(self.date_char, "%Y%m%d%H%M%S")
        LittleDataDict["slp"] = self.slp
        LittleDataDict["slp_qc"] = self.slp_qc
        LittleDataDict["ref_pres"] = self.ref_pres_qc
        LittleDataDict["ground_t"] = self.ground_t
        LittleDataDict["ground_t_qc"] = self.ground_t_qc
        LittleDataDict["psfc"] = self.psfc
        LittleDataDict["psfc_qc"] = self.psfc_qc
        LittleDataDict["precip"] = self.precip
        LittleDataDict["precip_qc"] = self.precip_qc
        LittleDataDict["t_max"] = self.t_max
        LittleDataDict["t_max_qc"] = self.t_max_qc
        LittleDataDict["t_min"] = self.t_min
        LittleDataDict["t_min_qc"] = self.t_min_qc
        LittleDataDict["t_min_night"] = self.t_min_night
        LittleDataDict["t_min_night_qc"] = self.t_min_night_qc
        LittleDataDict["cloud_cvr"] = self.cloud_cvr
        LittleDataDict["cloud_cvr_qc"] = self.cloud_cvr_qc
        LittleDataLevelDictlist = []
        if self.sound_layers > 0:
            for i in range(self.sound_layers):
                LittleDataLevelDict = {}
                LittleDataLevelDict["little_data_id"] = data_id
                LittleDataLevelDict["level"] = i
                LittleDataLevelDict["pres"] = self.sound_pressure[i]
                LittleDataLevelDict["height"] = self.sound_height[i]
                LittleDataLevelDict["temp"] = self.sound_temperature[i]
                LittleDataLevelDict["dew"] = self.sound_dew_point[i]
                LittleDataLevelDict["wspd"] = self.sound_speed[i]
                LittleDataLevelDict["dir"] = self.sound_direction[i]
                LittleDataLevelDictlist.append(LittleDataLevelDict)
        else:
            raise ValueError(
                'Input data error! Please check header, data and end!')
        LittleDataDict["level_info"] = json.dumps(LittleDataLevelDictlist)
        LittleData.objects.create(**LittleDataDict)

    def set_little_china_data(self):
        lp = LittlePoint.objects.raw("select * from littler_littlepoint_china where latitude=%s and longitude=%s" % (
        self.latitude, self.longitude))
        if lp.count() == 0:
            if 'China' in self.station_name or 'CHINA' in self.station_name:
                LittlePointDict = {}
                point_id = str(uuid.uuid1())
                LittlePointDict["point_id"] = point_id
                LittlePointDict["latitude"] = self.latitude
                LittlePointDict["longitude"] = self.longitude
                LittlePointDict["station_id"] = self.station_id
                LittlePointDict["station_name"] = self.station_name
                LittlePointDict["platform"] = self.platform
                # date_char = datetime.strptime(self.date_char, "%Y%m%d%H%M%S")
                # LittlePointDict["last_data_time"] = date_char
                # LittlePointDict["frist_data_time"] = date_char
                LittlePoint.objects.create(**LittlePointDict)
                # point = LittlePoint.objects.filter(latitude=self.latitude,longitude=self.longitude)
        else:
            point_id = lp[0].point_id
            # data_count = lp[0].data_count + 1
            # last_data_time = lp[0].last_data_time
            # frist_data_time = lp[0].frist_data_time
            # date_char = datetime.strptime(self.date_char, "%Y%m%d%H%M%S")
            # if frist_data_time == None or frist_data_time > date_char:
            #     frist_data_time = date_char
            # if last_data_time == None or last_data_time < date_char:
            #     last_data_time = date_char
            # lp.update(data_count=data_count, last_data_time=last_data_time, frist_data_time=frist_data_time)

        LittleDataDict = {}
        data_id = str(uuid.uuid1())
        LittleDataDict["data_id"] = data_id
        LittleDataDict["little_point_id"] = point_id
        LittleDataDict["source"] = self.source
        LittleDataDict["elevation"] = self.elevation
        LittleDataDict["num_valid_fields"] = self.num_valid_fields
        LittleDataDict["seq_num"] = self.seq_num
        LittleDataDict["num_dups"] = self.num_dups
        LittleDataDict["is_sound"] = self.is_sound
        LittleDataDict["bogus"] = self.bogus
        LittleDataDict["discard"] = self.discard
        LittleDataDict["sut"] = self.sut
        LittleDataDict["julian"] = self.julian
        LittleDataDict["date_char"] = datetime.strptime(self.date_char, "%Y%m%d%H%M%S")
        LittleDataDict["slp"] = self.slp
        LittleDataDict["slp_qc"] = self.slp_qc
        LittleDataDict["ref_pres"] = self.ref_pres_qc
        LittleDataDict["ground_t"] = self.ground_t
        LittleDataDict["ground_t_qc"] = self.ground_t_qc
        LittleDataDict["psfc"] = self.psfc
        LittleDataDict["psfc_qc"] = self.psfc_qc
        LittleDataDict["precip"] = self.precip
        LittleDataDict["precip_qc"] = self.precip_qc
        LittleDataDict["t_max"] = self.t_max
        LittleDataDict["t_max_qc"] = self.t_max_qc
        LittleDataDict["t_min"] = self.t_min
        LittleDataDict["t_min_qc"] = self.t_min_qc
        LittleDataDict["t_min_night"] = self.t_min_night
        LittleDataDict["t_min_night_qc"] = self.t_min_night_qc
        LittleDataDict["cloud_cvr"] = self.cloud_cvr
        LittleDataDict["cloud_cvr_qc"] = self.cloud_cvr_qc
        LittleDataLevelDictlist = []
        if self.sound_layers > 0:
            for i in range(self.sound_layers):
                LittleDataLevelDict = {}
                LittleDataLevelDict["little_data_id"] = data_id
                LittleDataLevelDict["level"] = i
                LittleDataLevelDict["pres"] = self.sound_pressure[i]
                LittleDataLevelDict["height"] = self.sound_height[i]
                LittleDataLevelDict["temp"] = self.sound_temperature[i]
                LittleDataLevelDict["dew"] = self.sound_dew_point[i]
                LittleDataLevelDict["wspd"] = self.sound_speed[i]
                LittleDataLevelDict["dir"] = self.sound_direction[i]
                LittleDataLevelDictlist.append(LittleDataLevelDict)
        else:
            raise ValueError(
                'Input data error! Please check header, data and end!')
        LittleDataDict["level_info"] = json.dumps(LittleDataLevelDictlist)
        LittleData.objects.create(**LittleDataDict)

class LittleRReader(object):

    def __init__(self, littler_file):
        self.littler_file = littler_file
        self.reports = []
        self.fid = None

    def read_report(self):
        header_format = '( 2f20.5, 2a40, 2a40, 1f20.5, 5i10, 3L10, 2i10, a20, 13(f13.5, i7) )'
        data_format = '( 10(f13.5, i7) )'
        end_format = '( 3 ( i7 ) )'

        header_reader = FortranRecordReader(header_format)
        data_reader = FortranRecordReader(data_format)
        end_reader = FortranRecordReader(end_format)

        # if not open fid, open it
        if self.fid is None:
            self.fid = open(self.littler_file, 'r')

        # read report header
        line = self.fid.readline()
        if not line:
            print('End of file')
            return None
        else:
            header = header_reader.read(line)

        # read data records
        data_flag = True
        data_recs = []
        while data_flag:
            line = self.fid.readline()
            if not line:
                print('End of file, report not complete')
                break
            else:
                data = data_reader.read(line)
                data_recs.append(data)
                #print('data: %s' % data)
                if data[0] == -777777.0:
                    data_flag = False

        # read report end
        line = self.fid.readline()
        if not line:
            print('End of file, report not complete')
            return None
        else:
            end = end_reader.read(line)
            #print('end: %s' % end)
        report = LittleRReport(header, data_recs, end)
        return report

    def read_all_reports(self):
        '''
        Read all reports from file.
        '''
        self.fid = open(self.littler_file, 'r')
        i = 0
        while True:
            report = self.read_report()
            if report is None:
                break
            else:
                i += 1
                print 'read %s report.' % i
                self.reports.append(report)
        self.fid.close()
        self.fid = None

def little_r_add_data():
    # 时间参数
    enddt = datetime.now() - timedelta(hours=1)
    begdt = datetime.now() - timedelta(hours=12)
    file_url = "%s%s/" % (settings.LITTER_R_FILE_URL, begdt.strftime("%Y"))
    dec_file_url = "%s%s/" % (settings.LITTER_R_DEC_FILE_SHELL_URL, begdt.strftime("%Y"))
    obs_finder = ObsDataFinder(dec_file_url)
    little_points = []
    obs_finder.batch_process(begdt.strftime("%Y%m%d%H"), enddt.strftime("%Y%m%d%H"), file_url,
                             little_points=little_points,
                             func=little_r_add_data_single)
    cursor = connections["littler"].cursor()
    little_points = set(little_points)

    for little_point in little_points:
        cursor.execute(
            "select public.update_littlepoint_singlecount('%s','%s','%s');" % (little_point,
                                                                               begdt.strftime(
                                                                                   "%Y-%m-%d %H:00:00"),
                                                                               enddt.strftime(
                                                                                   "%Y-%m-%d %H:59:59")))
        # transaction.set_dirty()
        # row = cursor.fetchone()
        transaction.commit()

def little_r_add_data_all(dirname):
    # LITTER_R_FILE_SHELL_URL
    begdtstr = "%s010100" % (dirname)
    enddtstr = "%s123123" % (dirname)
    file_url = "%s%s/" % (settings.LITTER_R_FILE_URL, dirname)
    dec_file_url = "%s%s/" % (settings.LITTER_R_DEC_FILE_SHELL_URL, dirname)
    obs_finder = ObsDataFinder(dec_file_url)
    little_points = []
    obs_finder.batch_process(begdtstr, enddtstr, file_url, little_points=little_points,
                             func=little_r_add_data_single)
    cursor = connections["littler"].cursor()
    little_points = set(little_points)
    begdt = datetime.strptime(begdtstr, "%Y%m%d%H")
    enddt = datetime.strptime(enddtstr, "%Y%m%d%H")
    print(little_points)
    for little_point in little_points:
        cursor.execute(
            "select public.update_littlepoint_singlecount('%s','%s','%s');" % (little_point,
                                                                               begdt.strftime(
                                                                                   "%Y-%m-%d %H:00:00"),
                                                                               enddt.strftime(
                                                                                   "%Y-%m-%d %H:59:59")))
        # transaction.set_dirty()
        transaction.commit()


def little_r_add_data_single(dest, obs_filename, little_points=[]):
    if len(LittleLog.objects.filter(filename=obs_filename)) == 0:
        LittleLogDir = {}
        LittleLogDir["filename"] = obs_filename
        data_dt = datetime.strptime(obs_filename, "obs:%Y-%m-%d_%H")
        LittleLogDir["datafile_dt"] = data_dt
        LittleLog.objects.create(**LittleLogDir)
        r_reader = LittleRReader("%s%s" % (dest, obs_filename))

        while True:
            try:
                report = r_reader.read_report()
                if report is None:
                    break
                else:
                    report.set_little_data()
                    little_points.append(report.point_id)
            except Exception as e:
                continue
        # transaction.commit_unless_managed()
        # cursor.execute("select public.update_littlepoint_count_time('%s','%s');" % (
        # data_dt.strftime("%Y-%m-%d %H:00:00"), data_dt.strftime("%Y-%m-%d %H:59:59")))
        # transaction.commit_unless_managed()
        RM("%s%s" % (dest, obs_filename))
    else:
        RM("%s%s" % (dest, obs_filename))
